{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload new NFIRS transaction file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing files\n",
      "Incidents loaded: 0\n"
     ]
    }
   ],
   "source": [
    "import psycopg2\n",
    "from nfirs_transaction_parser.settings import CONFIG\n",
    "\n",
    "# Override DB_STRING to production\n",
    "DB_STRING = 'postgresql://firecares:password@localhost/nfirs'\n",
    "\n",
    "CONFIG['sqlalchemy.url'] = DB_STRING\n",
    "conn = psycopg2.connect(DB_STRING)\n",
    "\n",
    "from nfirs_transaction_parser.core.parser import IncidentFile\n",
    "from nfirs_transaction_parser.core.loader import DBLoader\n",
    "\n",
    "\n",
    "# List of files to be processed\n",
    "# eg. ['newdep/data.inc.txt', 'newdep/data2.inc.txt',']\n",
    "FILES = []\n",
    "# FDID of the department to update\n",
    "FDID = \"66003\"\n",
    "STATE = 'OK'\n",
    "\n",
    "# Load files to DB\n",
    "print \"Parsing files\"\n",
    "incidents = 0\n",
    "for item in FILES:\n",
    "    if item:\n",
    "        inc = IncidentFile.load(open(item))\n",
    "        incidents += len(inc.incidents)\n",
    "        DBLoader(inc).load()\n",
    "print \"Incidents loaded: {}\".format(incidents)\n",
    "\n",
    "if incidents:\n",
    "    # Update materialized views inside department_nfirs schema\n",
    "    print \"Updating department_nfirs materialized views\"\n",
    "    with conn.cursor() as cursor:\n",
    "        cursor.execute(\"REFRESH MATERIALIZED VIEW department_nfirs.buildingfires\")\n",
    "        conn.commit()\n",
    "\n",
    "    # Update materialized views in the public schema\n",
    "    print \"Updating public materialized views\"\n",
    "    with conn.cursor() as cursor:\n",
    "        cursor.execute(\"REFRESH MATERIALIZED VIEW joint_buildingfires\")\n",
    "        cursor.execute(\"REFRESH MATERIALIZED VIEW joint_incidentaddress\")\n",
    "        cursor.execute(\"REFRESH MATERIALIZED VIEW joint_fireincident\")\n",
    "        cursor.execute(\"REFRESH MATERIALIZED VIEW joint_ffcasualty\")\n",
    "        cursor.execute(\"REFRESH MATERIALIZED VIEW joint_civiliancasualty\")\n",
    "        conn.commit()\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named firecares.settings.local",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-70d6e884506a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdjango\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconf\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mconf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'CELERY_ALWAYS_EAGER'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdjango\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdjango\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfirecares\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mupdate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/webapps/firecares/local/lib/python2.7/site-packages/django/utils/functional.pyc\u001b[0m in \u001b[0;36m__setattr__\u001b[0;34m(self, name, value)\u001b[0m\n\u001b[1;32m    252\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrapped\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 254\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    255\u001b[0m             \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrapped\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/webapps/firecares/local/lib/python2.7/site-packages/django/conf/__init__.pyc\u001b[0m in \u001b[0;36m_setup\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m     42\u001b[0m                 % (desc, ENVIRONMENT_VARIABLE))\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSettings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings_module\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/webapps/firecares/local/lib/python2.7/site-packages/django/conf/__init__.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, settings_module)\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSETTINGS_MODULE\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msettings_module\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m         \u001b[0mmod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_module\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSETTINGS_MODULE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         tuple_settings = (\n",
      "\u001b[0;32m/usr/lib/python2.7/importlib/__init__.pyc\u001b[0m in \u001b[0;36mimport_module\u001b[0;34m(name, package)\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0mlevel\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_resolve_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpackage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m     \u001b[0m__import__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named firecares.settings.local"
     ]
    }
   ],
   "source": [
    "import django.conf as conf\n",
    "setattr(conf.settings, 'CELERY_ALWAYS_EAGER', True)\n",
    "import django\n",
    "django.setup()\n",
    "from firecares.tasks import update\n",
    "from firecares import celery\n",
    "from firecares.firestation.models import FireDepartment\n",
    "fd = FireDepartment.objects.get(fdid=FDID, state=STATE)\n",
    "\n",
    "# Export buildingfires geom\n",
    "print \"Export buildingfires geom with the command:\"\n",
    "sql = \"\"\"\\COPY (select alarm, a.inc_type, alarms,ff_death, oth_death, ST_X(geom) as x, st_y(geom) as y, COALESCE(y.risk_category, 'Unknown') as risk_category from buildingfires a LEFT JOIN (SELECT state, fdid, inc_date, inc_no, exp_no, x.geom, x.parcel_id, x.risk_category FROM ( SELECT * FROM incidentaddress a LEFT JOIN parcel_risk_category_local using (parcel_id)) AS x) AS y USING (state, fdid, inc_date, inc_no, exp_no) WHERE a.state = '{state}' and a.fdid = '{fdid}') to PROGRAM 'aws s3 cp - s3://firecares-test/{id}-building-fires.csv --acl=\\\"public-read\\\"' DELIMITER ',' CSV HEADER;\"\"\"\n",
    "print sql.format(fdid=FDID, state=STATE, id=fd.id)\n",
    "update.update_department(fd.id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
